{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.99960625,  0.02806042,  0.9977344 , -0.06727564, -0.00378651,\n",
       "        0.08661526], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('Acrobot-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAR8AAAEXCAYAAACUBEAgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAdBElEQVR4nO3df1BU990v8PdZYBcQdhGV3TBCZB5zY3hUkqDBbf7o3LiVWJrmB5nbZhzLWCeZmNWJknEa2mimaefiNffmV2u0T9qqczsJfcxTk2piEi4abK7rL5QEMaF2rgYedRd/lF1A2IXdz/1DOXGVGBbhfHc379fMmXHP97PsZ1n27fl9NBEREBEZzKS6ASL6dmL4EJESDB8iUoLhQ0RKMHyISAmGDxEpwfAhIiUYPkSkBMOHiJRg+BCREsrCZ8OGDZg2bRrS09NRVlaGgwcPqmqFiBRQEj5//vOfUV1djeeffx5HjhxBSUkJysvL0dnZqaIdIlJAU3FiaVlZGebOnYvf/va3AIBIJIKCggKsWLECzz777Dc+PxKJ4MyZM8jOzoamaePdLhGNkIigu7sb+fn5MJluvGyTalBPulAohKamJtTU1OjzTCYTXC4XPB7PsM8JBoMIBoP649OnT6O4uHjceyWi0eno6MDUqVNvWGN4+Jw/fx7hcBh2uz1qvt1uxxdffDHsc2pra/HLX/7yuvkdHR2wWq3j0icRxS4QCKCgoADZ2dnfWGt4+IxGTU0Nqqur9cdDb9BqtTJ8iOLQSDaHGB4+kydPRkpKCnw+X9R8n88Hh8Mx7HMsFgssFosR7RGRQQzf22U2m1FaWoqGhgZ9XiQSQUNDA5xOp9HtEJEiSla7qqurUVVVhTlz5uCee+7BK6+8gt7eXixZskRFO0SkgJLw+dGPfoRz585h7dq18Hq9uPPOO/HBBx9ctxGaiJKXkuN8blYgEIDNZoPf7+cGZ6I4Est3k+d2EZESDB8iUoLhQ0RKMHyISAmGDxEpwfAhIiUYPkSkBMOHiJRg+BCREgwfIlKC4UNESjB8iEgJhg8RKcHwISIlGD5EpATDh4iUYPgQkRIMHyJSguFDREowfIhICYYPESnB8CEiJRg+RKQEw4eIlGD4EJESDB8iUoLhQ0RKMHyISAmGDxEpwfAhIiUYPkSkBMOHiJRg+BCREgwfIlKC4UNESsQcPnv37sUDDzyA/Px8aJqGd955J2pcRLB27VrccsstyMjIgMvlwokTJ6JqLl68iEWLFsFqtSInJwdLly5FT0/PTb0RIkosMYdPb28vSkpKsGHDhmHH169fj9deew2bNm3CgQMHMGHCBJSXl6O/v1+vWbRoEVpbW1FfX4+dO3di7969eOKJJ0b/Logo8chNACDbt2/XH0ciEXE4HPLiiy/q87q6usRischbb70lIiLHjx8XAHLo0CG9ZteuXaJpmpw+fXpEr+v3+wWA+P3+m2mfiMZYLN/NMd3mc/LkSXi9XrhcLn2ezWZDWVkZPB4PAMDj8SAnJwdz5szRa1wuF0wmEw4cODDszw0GgwgEAlETESW2MQ0fr9cLALDb7VHz7Xa7Pub1epGXlxc1npqaitzcXL3mWrW1tbDZbPpUUFAwlm0TkQIJsberpqYGfr9fnzo6OlS3REQ3aUzDx+FwAAB8Pl/UfJ/Pp485HA50dnZGjQ8ODuLixYt6zbUsFgusVmvURESJbUzDp6ioCA6HAw0NDfq8QCCAAwcOwOl0AgCcTie6urrQ1NSk1+zevRuRSARlZWVj2Q4RxbHUWJ/Q09ODf/zjH/rjkydPorm5Gbm5uSgsLMTKlSvx61//GrfddhuKioqwZs0a5Ofn46GHHgIA3HHHHbj//vvx+OOPY9OmTRgYGMDy5cvx4x//GPn5+WP2xogozsW6K23Pnj0C4LqpqqpKRC7vbl+zZo3Y7XaxWCwyf/58aWtri/oZFy5ckMcee0yysrLEarXKkiVLpLu7e8Q9cFc7UXyK5bupiYgozL5RCQQCsNls8Pv93P5DFEdi+W4mxN4uIko+DB8iUoLhQ0RKMHyISAmGDxEpwfAhIiUYPkSkBMOHiJSI+fSKePLpp58iKytLdRtEdEUsl0NO6PA5d+4cLl26pLoNIrqit7d3xLUJHT4ul4unVxDFkViuMsptPkSkBMOHiJRg+BCREgwfIlKC4UNESjB8iEgJhg8RKcHwISIlGD5EpATDh4iUYPgQkRIMHyJSguFDREowfIhICYYPESnB8CEiJRg+RKQEw4eIlGD4EJESDB8iUoLhQ0RKMHyISAmGDxEpwfAhIiViCp/a2lrMnTsX2dnZyMvLw0MPPYS2traomv7+frjdbkyaNAlZWVmorKyEz+eLqmlvb0dFRQUyMzORl5eH1atXY3Bw8ObfDREljJjCp7GxEW63G/v370d9fT0GBgawYMGCqFukrlq1Cjt27MC2bdvQ2NiIM2fO4JFHHtHHw+EwKioqEAqFsG/fPmzduhVbtmzB2rVrx+5dEVH8k5vQ2dkpAKSxsVFERLq6uiQtLU22bdum13z++ecCQDwej4iIvP/++2IymcTr9eo1GzduFKvVKsFgcESv6/f7BYD4/f6baZ+Ixlgs382b2ubj9/sBALm5uQCApqYmDAwMwOVy6TUzZsxAYWEhPB4PAMDj8WDWrFmw2+16TXl5OQKBAFpbW4d9nWAwiEAgEDURUWIbdfhEIhGsXLkS9957L2bOnAkA8Hq9MJvNyMnJiaq12+3wer16zdXBMzQ+NDac2tpa2Gw2fSooKBht20QUJ0YdPm63G8eOHUNdXd1Y9jOsmpoa+P1+fero6Bj31ySi8ZU6mictX74cO3fuxN69ezF16lR9vsPhQCgUQldXV9TSj8/ng8Ph0GsOHjwY9fOG9oYN1VzLYrHAYrGMplUiilMxLfmICJYvX47t27dj9+7dKCoqihovLS1FWloaGhoa9HltbW1ob2+H0+kEADidTrS0tKCzs1Ovqa+vh9VqRXFx8c28FyJKIDEt+bjdbrz55pt49913kZ2drW+jsdlsyMjIgM1mw9KlS1FdXY3c3FxYrVasWLECTqcT8+bNAwAsWLAAxcXFWLx4MdavXw+v14vnnnsObrebSzdE3yax7EYDMOy0efNmvaavr0+eeuopmThxomRmZsrDDz8sZ8+ejfo5p06dkoULF0pGRoZMnjxZnnnmGRkYGBhxH9zVThSfYvluaiIi6qJvdAKBAGw2G/x+P6xWq+p2iOiKWL6bPLeLiJRg+BCREgwfIlKC4UNESjB8iEgJhg8RKcHwISIlGD5EpATDh4iUGNVZ7URjQSSMcNiPSKQPmpaKlBQrNC0dmqapbo0MwPAhw4kIBgZO49y5TfD7dyAU6oDJNAGZmXOQl/cUsrP/KzSNf5rJjp8wGUpEEAyewKlTS9Db68Hlc5OBcPif8Pv/Ez09e5Cf/98xZcoTDKAkx20+ZKhw2I/29hXo7d2HoeC5dvz06Z/D7/8ACXjOM8WA4UOG8vvfQ3d3ww1rIhE/fL7/hUik94Z1lNgYPmSonp5PAIS/se7SpSMIh3mXkmTG8CHDBMJhfNbXN+J6rnQlN4YPGSYQDuNgnzaiUOlDOvqFu9yTGcOHDPUeKhDAja9wJwB24X6cE5sxTZESDB8y1P9DEf4NT6Afw98sQAAcxV34ExZxySfJ8UAKMlQEJvwHKgEAVdiKSTgP05UVsRDM8GAeXsEq/BNTEIpEVLZK44zhQ4YbRBr+Hf8NHjgxD/sxFf+JXkzApyjBpyhBHzKRCSDE43ySGsOHlBCY0I5b0Y5bhx8XYfgkOW7zobgk4JJPsmP4UFyKAOgKf/PBiJS4GD5kmAkmE/5lhLfEDongWAwHJFLiYfiQYdI0DRNTUlS3QXGC4UOG0XA5gIgAhg8ZyKRpDB/SMXzIMFzyoasxfMgwJgBmE//k6DL+JZBhuNpFV2P4kGG42kVXY/iQYUyIPXx4HefkxfAhQ8VylE+YwZPUGD5kmFhvBhgS4aVUkxjDh+IWwye5xRQ+GzduxOzZs2G1WmG1WuF0OrFr1y59vL+/H263G5MmTUJWVhYqKyvh8/mifkZ7ezsqKiqQmZmJvLw8rF69GoODg2PzbiipMHySW0zhM3XqVKxbtw5NTU04fPgw7rvvPjz44INobW0FAKxatQo7duzAtm3b0NjYiDNnzuCRRx7Rnx8Oh1FRUYFQKIR9+/Zh69at2LJlC9auXTu274qSAq9kmNw0ucndCbm5uXjxxRfx6KOPYsqUKXjzzTfx6KOPAgC++OIL3HHHHfB4PJg3bx527dqFH/zgBzhz5gzsdjsAYNOmTfjZz36Gc+fOwWw2D/sawWAQwWBQfxwIBFBQUAC/3w+r9cYXI6f4suzLL7Hp/PkR1f4kNxe/nzaNu+cTSCAQgM1mG9F3c9TbfMLhMOrq6tDb2wun04mmpiYMDAzA5XLpNTNmzEBhYSE8Hg8AwOPxYNasWXrwAEB5eTkCgYC+9DSc2tpa2Gw2fSooKBht25RAQiLc1Z7EYg6flpYWZGVlwWKx4Mknn8T27dtRXFwMr9cLs9mMnJycqHq73Q6v1wsA8Hq9UcEzND409nVqamrg9/v1qaOjI9a2KQENcJtPUov5Gs633347mpub4ff78fbbb6OqqgqNjY3j0ZvOYrHAMsKLUFF8mxDDuV3+cBjc6pO8Yl7yMZvNmD59OkpLS1FbW4uSkhK8+uqrcDgcCIVC6Orqiqr3+XxwOBwAAIfDcd3er6HHQzWU3O7KzBzxgYat/f0IcqNz0rrp43wikQiCwSBKS0uRlpaGhoYGfaytrQ3t7e1wOp0AAKfTiZaWFnR2duo19fX1sFqtKC4uvtlWKAFYuPGYrohptaumpgYLFy5EYWEhuru78eabb+Ljjz/Ghx9+CJvNhqVLl6K6uhq5ubmwWq1YsWIFnE4n5s2bBwBYsGABiouLsXjxYqxfvx5erxfPPfcc3G43V6u+JXhJDRoSU/h0dnbiJz/5Cc6ePQubzYbZs2fjww8/xPe+9z0AwMsvvwyTyYTKykoEg0GUl5fj9ddf15+fkpKCnTt3YtmyZXA6nZgwYQKqqqrwwgsvjO27orhl1jRw2YeAMTjOR4VYjiWg+LInEMCCEycwkmPab0lLw/HiYuSk8t6WicKQ43yIRoOrXTSEfwlkKIumxXx2OyUnhg8ZyszgoSsYPmQonqdFQxg+ZKhYVrkiIhhIvP0hNEIMH4pbAjB8khjDh+JWBJfPbKfkxPChuCVc7UpqDB+KW1ztSm4MH4pbETB8khnDh+KWiHCbTxJj+FDcCgPo5fV8khbDhwyVbTIhPy1tRLU9kQiO9/WNc0ekCsOHDJVpMmFyDGepc7kneTF8yFAmTUMqT7EgjOIC8kQ3wwTgq5UuQSoGMQ2nMA0nkYowTiMfJ/Bf0I90gJcdS2oMHzJUCnBlyUdghxeP4w18F3thRQAaBJeQic8wC5uwDMfB63onM4YPGcqkaUjTNEzBOTyPFzAHh2G66u5cWeiFE/sxFaexBi8AKFTXLI0rbvMhQ5kAmDXB43gDc3EoKniGaAAK0IFqvIwJ6DW8RzIGl3zIUCZNw3StHS78nxtu0dEAFOM4cuABUGRQd2QkLvmQoVIAWLU+ZKHnG2vTMIhcXBz/pkgJhg8ZSgDwDuwEMHzIYGEAnREbLiL3G2v7YcFp5I9/U6QEw4cMFRbBKbkFf8UPEb7Bn58AOIS5aMIc45ojQzF8yFBhEQQF+N9YjHq4hg0gAdCKf8UrWHnlYENKRtzbRYYK4/KlUbuRjf+Bn+FL3IrvYxfs8MGEMP6JXPxffAebsQSnMVV1uzSOGD5kqMtLPgJAQzeseAOPYzsehgM+mBDBBUzCWdyCCEzg6RXJjeFDhroUieDC4NCd2jUAGs4jD+eRN2w9twskL362ZKjuSARnBgZGVJtlMuFfMzLGuSNSheFDcStF05Bt4p9osuInS3HLBMDC8Ela/GQpbpkAWHjhsaTF8KG4ZdI0LvkkMX6yFLe45JPcGD5kqMEY7sNl0jSYGT5J66bCZ926ddA0DStXrtTn9ff3w+12Y9KkScjKykJlZSV8Pl/U89rb21FRUYHMzEzk5eVh9erVGNSP/aBk1sf7cNEVow6fQ4cO4Xe/+x1mz54dNX/VqlXYsWMHtm3bhsbGRpw5cwaPPPKIPh4Oh1FRUYFQKIR9+/Zh69at2LJlC9auXTv6d0EJoz8SgfAupIRRhk9PTw8WLVqEN954AxMnTtTn+/1+/OEPf8BLL72E++67D6Wlpdi8eTP27duH/fv3AwA++ugjHD9+HH/6059w5513YuHChfjVr36FDRs2IBQKDft6wWAQgUAgaqLE1MfgoStGFT5utxsVFRVwuVxR85uamjAwMBA1f8aMGSgsLITH4wEAeDwezJo1C3a7Xa8pLy9HIBBAa2vrsK9XW1sLm82mTwUFBaNpm+JAfyTCS4kRgFGET11dHY4cOYLa2trrxrxeL8xmM3JycqLm2+12eL1evebq4BkaHxobTk1NDfx+vz51dHTE2jbFCW7zoSExnVja0dGBp59+GvX19UhPN+46KxaLBRaLxbDXo/HzZSg04lsgpwDQuLcracW05NPU1ITOzk7cfffdSE1NRWpqKhobG/Haa68hNTUVdrsdoVAIXV1dUc/z+XxwOBwAAIfDcd3er6HHQzWUvE4GgyNe7bozM5PH+SSxmMJn/vz5aGlpQXNzsz7NmTMHixYt0v+dlpaGhoYG/TltbW1ob2+H0+kEADidTrS0tKCzs1Ovqa+vh9VqRXEx71BJX8k2mXggWhKLabUrOzsbM2fOjJo3YcIETJo0SZ+/dOlSVFdXIzc3F1arFStWrIDT6cS8efMAAAsWLEBxcTEWL16M9evXw+v14rnnnoPb7eaqFUWxmExc7UpiY34xsZdffhkmkwmVlZUIBoMoLy/H66+/ro+npKRg586dWLZsGZxOJyZMmICqqiq88MILY90KJbh0TeO1DJOYJgl4xFcgEIDNZoPf74fValXdDsVg2ZdfYtP58yOqXZmXh/85dSpSuPSTMGL5bnKVmgwT6/9zXPJJbgwfMlQ4hlqzycTwSWIMHzJMBEAwhoMMNfA4n2TG8CHDfHXbHCKGDxko1iUfSm4MHzJMBOCSD+kYPmSYCFe76CoMHzJMfySCjq+5ZtNw0rixOakxfMgw/SLwjvBupRZNQwnvVprUGD4Ul0wAJvC2OUmNny7FJU3TkM7wSWr8dCkuaQDDJ8nx06W4pOHyuV2UvBg+FJe45JP8+OmSYcIxHOOjAbxbaZJj+JBhgiIx3TaH0ZPcGD5kGN6zi67G8CHD9Me45EPJjeFDhuF92ulqDB8yDFe76GoMHzLMiWAQAyNc8rGYTDBxb1dSY/iQYbwDAyO+hvMd6ek8tyvJ8dOluJRhMvGWOUmO4UNxyaxp/ONMcvx8KS5ZGD5Jj58vxaV0bnBOegwfiksW3q006aWqboC+PSonTkSh2YxLkQh6r57C4evm/YvFwv8ZkxzDhwxTkpERdV3ma4/4ufox71aa/Bg+ZJhrw4TR8u3GJVsiUoLhQ0RKMHyISAmGDxEpwfAhIiUYPkSkBMOHiJRIyON8hi7FGQgEFHdCRFcb+k6O5HK5CRk+Fy5cAAAUFBQo7oSIhtPd3Q2bzXbDmoQMn9zcXABAe3v7N77BeBMIBFBQUICOjg5YrVbV7YwY+zZWovYtIuju7kZ+fv431iZk+JiuXF7TZrMl1AdzNavVmpC9s29jJWLfI10g4AZnIlKC4UNESiRk+FgsFjz//POwWCyqW4lZovbOvo2VqH3HQhPeQpKIFEjIJR8iSnwMHyJSguFDREowfIhICYYPESmRkOGzYcMGTJs2Denp6SgrK8PBgweV9rN371488MADyM/Ph6ZpeOedd6LGRQRr167FLbfcgoyMDLhcLpw4cSKq5uLFi1i0aBGsVitycnKwdOlS9PT0jGvftbW1mDt3LrKzs5GXl4eHHnoIbW1tUTX9/f1wu92YNGkSsrKyUFlZCZ/PF1XT3t6OiooKZGZmIi8vD6tXr8bg4OC49b1x40bMnj1bP/rX6XRi165dcd3zcNatWwdN07By5cqE631MSIKpq6sTs9ksf/zjH6W1tVUef/xxycnJEZ/Pp6yn999/X37xi1/IX/7yFwEg27dvjxpft26d2Gw2eeedd+TTTz+VH/7wh1JUVCR9fX16zf333y8lJSWyf/9++dvf/ibTp0+Xxx57bFz7Li8vl82bN8uxY8ekublZvv/970thYaH09PToNU8++aQUFBRIQ0ODHD58WObNmyff+c539PHBwUGZOXOmuFwuOXr0qLz//vsyefJkqampGbe+//rXv8p7770nf//736WtrU1+/vOfS1pamhw7dixue77WwYMHZdq0aTJ79mx5+umn9fmJ0PtYSbjwueeee8TtduuPw+Gw5OfnS21trcKuvnJt+EQiEXE4HPLiiy/q87q6usRischbb70lIiLHjx8XAHLo0CG9ZteuXaJpmpw+fdqw3js7OwWANDY26n2mpaXJtm3b9JrPP/9cAIjH4xGRy8FrMpnE6/XqNRs3bhSr1SrBYNCw3idOnCi///3vE6Ln7u5uue2226S+vl6++93v6uGTCL2PpYRa7QqFQmhqaoLL5dLnmUwmuFwueDwehZ19vZMnT8Lr9Ub1bLPZUFZWpvfs8XiQk5ODOXPm6DUulwsmkwkHDhwwrFe/3w/gq6sGNDU1YWBgIKr3GTNmoLCwMKr3WbNmwW636zXl5eUIBAJobW0d957D4TDq6urQ29sLp9OZED273W5UVFRE9Qgkxu97LCXUWe3nz59HOByO+sUDgN1uxxdffKGoqxvzer0AMGzPQ2Nerxd5eXlR46mpqcjNzdVrxlskEsHKlStx7733YubMmXpfZrMZOTk5N+x9uPc2NDZeWlpa4HQ60d/fj6ysLGzfvh3FxcVobm6O254BoK6uDkeOHMGhQ4euG4vn3/d4SKjwofHjdrtx7NgxfPLJJ6pbGZHbb78dzc3N8Pv9ePvtt1FVVYXGxkbVbd1QR0cHnn76adTX1yM9PV11O8ol1GrX5MmTkZKSct3Wf5/PB4fDoairGxvq60Y9OxwOdHZ2Ro0PDg7i4sWLhryv5cuXY+fOndizZw+mTp2qz3c4HAiFQujq6rph78O9t6Gx8WI2mzF9+nSUlpaitrYWJSUlePXVV+O656amJnR2duLuu+9GamoqUlNT0djYiNdeew2pqamw2+1x2/t4SKjwMZvNKC0tRUNDgz4vEomgoaEBTqdTYWdfr6ioCA6HI6rnQCCAAwcO6D07nU50dXWhqalJr9m9ezcikQjKysrGrTcRwfLly7F9+3bs3r0bRUVFUeOlpaVIS0uL6r2trQ3t7e1Rvbe0tESFZ319PaxWK4qLi8et92tFIhEEg8G47nn+/PloaWlBc3OzPs2ZMweLFi3S/x2vvY8L1Vu8Y1VXVycWi0W2bNkix48flyeeeEJycnKitv4brbu7W44ePSpHjx4VAPLSSy/J0aNH5csvvxSRy7vac3Jy5N1335XPPvtMHnzwwWF3td91111y4MAB+eSTT+S2224b913ty5YtE5vNJh9//LGcPXtWny5duqTXPPnkk1JYWCi7d++Ww4cPi9PpFKfTqY8P7fpdsGCBNDc3ywcffCBTpkwZ112/zz77rDQ2NsrJkyfls88+k2effVY0TZOPPvoobnv+Olfv7Uq03m9WwoWPiMhvfvMbKSwsFLPZLPfcc4/s379faT979uwRANdNVVVVInJ5d/uaNWvEbreLxWKR+fPnS1tbW9TPuHDhgjz22GOSlZUlVqtVlixZIt3d3ePa93A9A5DNmzfrNX19ffLUU0/JxIkTJTMzUx5++GE5e/Zs1M85deqULFy4UDIyMmTy5MnyzDPPyMDAwLj1/dOf/lRuvfVWMZvNMmXKFJk/f74ePPHa89e5NnwSqfebxev5EJESCbXNh4iSB8OHiJRg+BCREgwfIlKC4UNESjB8iEgJhg8RKcHwISIlGD5EpATDh4iUYPgQkRL/H7u75KIKr3hVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([ -1.        -1.        -1.        -1.       -12.566371 -28.274334], [ 1.        1.        1.        1.       12.566371 28.274334], (6,), float32)\n",
      "env.action_space= Discrete(3)\n",
      "state= [ 0.99699575 -0.07745624  0.99999446 -0.00333461 -0.01623835  0.09868391]\n",
      "action= 1\n",
      "next_state= [ 0.9974679  -0.07111807  0.999961    0.00882816  0.07877667  0.02199956]\n",
      "reward= -1.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.a2c.a2c.A2C at 0x7f5ee4dd5d90>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import A2C\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = A2C(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=5,  #运行N步后执行更新,batch_size=n_steps*环境数量\n",
    "    gamma=0.9,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-200.0, 0.0)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(100_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/1.A2C.Acrobot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-84.55, 30.513070969668064)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = A2C.load('save/1.A2C.Acrobot')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAR8AAAEXCAYAAACUBEAgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAhE0lEQVR4nO3df3BU9d0v8PfZn/mx2d0EyK4pieQRKqb8UAOErbV6S0q0qZVCb5XL2NQyOtLAgDjcmlbx1tt5wuC0VluFzjgV7r0Ppg+2YKGgzRMw1IeVH4Fg+BVppSYSdgPE/ZHd7O/v/UNyHhci7EKSbza8XzNnxpzvZ7Ofg+ybc8737DmKEEKAiGiYaWQ3QEQ3JoYPEUnB8CEiKRg+RCQFw4eIpGD4EJEUDB8ikoLhQ0RSMHyISAqGDxFJIS18XnnlFUyYMAFZWVmoqKjA/v37ZbVCRBJICZ8//OEPWLlyJZ577jkcOnQI06dPR1VVFbq7u2W0Q0QSKDK+WFpRUYGZM2fit7/9LQAgkUiguLgYy5Ytw9NPP33V1ycSCXR1dSEvLw+Kogx1u0SUIiEE/H4/ioqKoNFced9GN0w9qSKRCFpaWlBXV6eu02g0qKyshNPpHPA14XAY4XBY/fnMmTMoKysb8l6J6Np0dnZi/PjxV6wZ9vA5f/484vE4bDZb0nqbzYaTJ08O+Jr6+nr8/Oc/v2x9Z2cnzGbzkPRJROnz+XwoLi5GXl7eVWuHPXyuRV1dHVauXKn+3L+BZrOZ4UM0AqVyOmTYw2fs2LHQarVwu91J691uN+x2+4CvMRqNMBqNw9EeEQ2TYZ/tMhgMKC8vR1NTk7oukUigqakJDodjuNshIkmkHHatXLkSNTU1mDFjBmbNmoVf//rXCAQCePTRR2W0Q0QSSAmfhx56COfOncPq1avhcrlw++234+23377sJDQRjV5SrvO5Xj6fDxaLBV6vlyeciUaQdD6b/G4XEUnB8CEiKRg+RCQFw4eIpGD4EJEUDB8ikoLhQ0RSMHyISAqGDxFJwfAhIikYPkQkBcOHiKRg+BCRFAwfIpKC4UNEUjB8iEgKhg8RScHwISIpGD5EJAXDh4ikYPgQkRQMHyKSguFDRFIwfIhICoYPEUnB8CEiKRg+RCQFw4eIpGD4EJEUDB8ikoLhQ0RSMHyISAqGDxFJwfAhIikYPkQkRdrhs2fPHjzwwAMoKiqCoijYunVr0rgQAqtXr8ZNN92E7OxsVFZW4tSpU0k1PT09WLRoEcxmM6xWKxYvXoze3t7r2hAiyixph08gEMD06dPxyiuvDDi+du1avPzyy1i/fj327duH3NxcVFVVIRQKqTWLFi3CsWPH0NjYiO3bt2PPnj14/PHHr30riCjziOsAQGzZskX9OZFICLvdLl544QV1ncfjEUajUbzxxhtCCCGOHz8uAIgDBw6oNTt37hSKoogzZ86k9L5er1cAEF6v93raJ6JBls5nc1DP+Zw+fRoulwuVlZXqOovFgoqKCjidTgCA0+mE1WrFjBkz1JrKykpoNBrs27dvwN8bDofh8/mSFiLKbIMaPi6XCwBgs9mS1ttsNnXM5XKhsLAwaVyn06GgoECtuVR9fT0sFou6FBcXD2bbRCRBRsx21dXVwev1qktnZ6fslojoOg1q+NjtdgCA2+1OWu92u9Uxu92O7u7upPFYLIaenh615lJGoxFmszlpIaLMNqjhU1paCrvdjqamJnWdz+fDvn374HA4AAAOhwMejwctLS1qza5du5BIJFBRUTGY7RDRCKZL9wW9vb34+9//rv58+vRptLa2oqCgACUlJVixYgV+8YtfYNKkSSgtLcWzzz6LoqIizJs3DwBw22234b777sNjjz2G9evXIxqNYunSpXj44YdRVFQ0aBuWCYQQiAqB9nAYTT4fFABLCwuhVRTZrRENvXSn0nbv3i0AXLbU1NQIIT6bbn/22WeFzWYTRqNRzJkzR7S3tyf9jgsXLoiFCxcKk8kkzGazePTRR4Xf70+5h0yeak8kEiIUj4sjgYB40eUS9548KQoOHxY4eFAUHTkiuiIR2S0SXbN0PpuKEEJIzL5r4vP5YLFY4PV6M+L8jxACISHwj3AY/+HzYZvXiyPBIC7E40l1BkXBptJSLMjPl9Qp0fVJ57OZ9mEXpW+b14uXu7txOBjEp/E4vijtI0KgyefDg1YrdDz0olEuI6baM92ZaBS7/X70XCF4+v3Z64X3kj0iotGI4TMM5uTlIU+rTan2XCyGd/3+Ie6ISD6GzzC42WCAIzc3pdqIEDgUDCKReafiiNLC8BkGBkXBtOzslOu3e70IJhJD2BGRfAyfYaAoChbk56d8dv90OIzWYBAZOBFJlDKGzzD5SlYW7szJSanWn0hgN2+uRqMcw2eY5Gq1uDcvL+X6rR4PwtzzoVEso6/zOXLkCEwmk+w2UnZzKAQoCqDXX7W2PRTCH48eRRmn3SmDpHM75IwOn3PnziEYDMpuI2U36/WYaDbj71cvRSCRwNauLozVZfT/IrrBBAKBlGsz+m92ZWVlRny9op8QArs/+QS/vOSWIl+k65ZbcO+XvwyjhkfHlBnSucso/1YPs1m5uUj1ixOHgkEc7esb0n6IZGH4DCNFUXBPXh5KDIaU6kNCYCfvV02jFMNnmI3R6VK+2hkAdvn9vOCQRiWGzzDTArjPbEZq3/QC9vb24qNweChbIpKC4TPMFEVBlcWS8ixWRAj88dNPebUzjToMHwnytVrcneL1SQLAkb4+RBg+NMowfCQwKAq+kZeX8qHXHr8fZ6PRIe2JaLgxfCRQFAWVZjNyUrx+xxuPo8nv56EXjSoMH0lsej2mpnibjRiAJp8PUYYPjSIMH0nyNJq0vmi6y+/H+VhsCDsiGl4MH0kURcH38/NhSPFG8RdiMfwHD71oFGH4SDTBaEz5Hj8xAI0+H/gddxotGD4SmTUafC2NW4I0+/34lIdeNEowfCRSFAXzrNaUby1wJhrFLh560SjB8JHsK1lZmJLirFcCwFsez1Wf/UWUCRg+klm0Wnw5Kyvl+v3BIM7x0ItGAYaPZIqi4JGCgpTr/xEOY18gwEMvyngMnxHgzpwc/EuK9/gBgH/r6RnCboiGB8NnBLDr9SlPuQPAyVAIPt5YnjIcw2cEUAAsyM9P+faqJ/r68AFvr0oZjuEzAiiKgq+ZTPhSCo/UAYAoLs568bwPZTCGzwhh0+sxM43bq+7p7YWXh16UwRg+I4QOwD1pXO3c2teHU7y9KmUwhs8IoVy8wVi+NrVbjEWFwGbeXpUyWFrhU19fj5kzZyIvLw+FhYWYN28e2tvbk2pCoRBqa2sxZswYmEwmLFiwAG63O6mmo6MD1dXVyMnJQWFhIVatWoUYL5zDLVlZaV1wuMfv56EXZay0wqe5uRm1tbV4//330djYiGg0irlz5yY9IvXJJ5/Etm3bsHnzZjQ3N6Orqwvz589Xx+PxOKqrqxGJRLB3715s3LgRGzZswOrVqwdvqzJUtqLg2xZLyvVtoRBORyJD2BHR0FHEdey3nzt3DoWFhWhubsbXv/51eL1ejBs3Dps2bcL3vvc9AMDJkydx2223wel0Yvbs2di5cye+/e1vo6urCzabDQCwfv16/OQnP8G5c+dgSOFiO5/PB4vFAq/Xm1GPS05FazCIu9vb0Zvis7qesdvxfFERlBTvC0Q0lNL5bF7XOR+v1wsAKLj49YCWlhZEo1FUVlaqNZMnT0ZJSQmcTicAwOl0YurUqWrwAEBVVRV8Ph+OHTs24PuEw2H4fL6kZbQqNRpRlsah106fL+WgIhpJrjl8EokEVqxYgbvuugtTpkwBALhcLhgMBlit1qRam80Gl8ul1nw+ePrH+8cGUl9fD4vFoi7FxcXX2vaIZ07z9qqnQiH8nbNelIGuOXxqa2tx9OhRNDQ0DGY/A6qrq4PX61WXzs7OIX9PWRRFwYNWa8q3V/UlEnjH5+OsF2WcawqfpUuXYvv27di9ezfGjx+vrrfb7YhEIvB4PEn1brcbdrtdrbl09qv/5/6aSxmNRpjN5qRlNJtgMOCmFK92BoDNn36KEMOHMkxa4SOEwNKlS7Flyxbs2rULpaWlSePl5eXQ6/VoampS17W3t6OjowMOhwMA4HA40NbWhu7ubrWmsbERZrMZZWVl17Mto4Zdr8fX07jg8Ewkgi7OelGGSfUOngA+O9TatGkT3nrrLeTl5annaCwWC7Kzs2GxWLB48WKsXLkSBQUFMJvNWLZsGRwOB2bPng0AmDt3LsrKyvDII49g7dq1cLlceOaZZ1BbWwuj0Tj4W5iBNIqChQUF+H89PZfdtVCPCEzwQ48YIjDAjzx0x4Ddfj/+xWjkrBdljLSm2r/oL/brr7+OH/7whwA+u8jwqaeewhtvvIFwOIyqqiq8+uqrSYdUH3/8MZYsWYJ3330Xubm5qKmpwZo1a6DTpZaFo3mqvd8nkQjubm/HPy/u0ShI4DYcx8NowO04AjN8+BT5OICZeAMP40u507DmS19CqdGIsTodDCk+DZVoMKXz2byu63xkuRHCJyEEHvroI7zp8UBBAnfjb3gaazAO55JuvSEAdKAE/xvP4oRyJ8br9ZiUlQVHbi6+ajJhotGIIr0e+ov/cHDPiIYSw2eU2Pzpp/j+Rx9hIk7hRTwJO1wD3vNHAPgHbsEK/Bou2IGLVRoARXo9bjEaUZGbC0duLm7LzkaJwQCjokABw4gGVzqfzbTO+dDwmpmTg5v1WjwU/QNuwsDXQAGfRc1E/APzsBXr8YS6PgHgk2gUn0SjaO7thRbAWJ0ON+n1+KrJhIrcXEzLzsYtRiOyNRpowTCi4cPwGcGKDQZMMOhwe7Q1pfo7ceiK43EA7lgM7lgMrX19WH/uHKxaLW7S6zEjJwezcnNxR04OJhmNyNNqYVAUhhENGYbPCKYBsKigAAhctfSaJAD0xOPoicdxLBTC/+3pQa5GA6tWi5kXg2h2bi7KsrJg1WqRrdEwjGjQMHxGMEVR4DCZ8E+tBak8pL0XqV8bNJAEAH8iAX8igU6PB1s8HhgVBQU6HaZmZ+OO7Gw4TCZ8JSsLhXo9TAwjug4MnxGuLDsPCdsSRLuWAvjiex6FYcCb+N6gvrcAEBICXdEouqJRvOPzQet2I//igw6nZmfjnrw8TM3ORrHBAPPnpvcZSnQ1DJ8RTqMo+ErhInSE3kNPzyZ8tn+SLAot/h3/HfsxC0j5GRjXJg7gfDyO84EA9gYC+N3587BotSg1GDAtOxtfM5kwPScHtxiNsGq10IBBRANj+GQArdaE8eN/BY3GhJ6ef0Ms0QsFAgIKfDDjj5iPjfghYkj9+2CDyRuPo7WvD619ffg/PT0wazSw6/Uoy87G3SYT7sjJQVlWFvK1Wug4xU8X8TqfDJJIhHChtwWHz7+Dj4Jn0Bq1Yk+iHB/iFsRH8L8juRoNxuh0+EpWFipyc1Gek4Op2dkYq9MhS6OBlkE0avAiw1EucvGk8EcXn9t+MBDA+4EA3LEYfPH4AAdmI0uWosCk1aLMqMe9uQJ35+pwR04e8g35UJQs7hVlMF5kOMoZNBqMubg3MSMnB7Fx4xC8GEb7AwG0BIM4EAigMxLBpyMwjEIigbyYG2WxN3FbYA+ylHP4UJOLcaZZKCz8MfLy/hsUhX81Rzvu+YwyQggkAIQuhlFbXx/+MxDA/kAAH0ciOB+LXfZN+WHuEMXowP/CzzEVbdBc0o1Wa0FR0b9i3LjHGUAZiHs+NzBFUaAFkKvVYmpODqZkZ2NhQQFCQuDjcBjHQyHs6e3FgUAAp8LhYQ8jE3rxP/ECpuGDAefl4nEvzpz5KQyGElgs1TwEG8UYPqNc/4c3W1EwOTsbk7OzMT8/H8F4HF3R6Gdh5PfjUDCIo6EQPLEYokPYz9fwHmZh/xUvCEgkvHC7f4m8vHuh1V7fhZM0cjF8blA5Wi0marWYmJWFBywWBBMJdMdiahh90NeHD/r60BOLISzEoO0d3Y5WaFM4CxUMHkI87mP4jGIMH4KiKMjValGq1aLUaMS3zGb0CQFPLIZjfX3YHwxifyCA1r4+XIjFEEwkJJ83otGA4UOXURQFOYqCHIMBRQYDKs1mhIWALx7Hsb4+HOrrw97eXhwJBtEdi8GfxnPDgsiBwNWvw9Zqc6EoqT23njITw4euSlEUZCkKsjQajNPpcG9eHpYXFsIbj+NEKITDwSD2+P1o6+vDJ9EoAlcIo7+gGg/iLVhwpQc/KsjPXwidbuzgbwyNGJxqp+smLp4T8sTjOBUOozUYxC6/HydDIXwUDiPwucM0BQl8H/+OZfgNsjDwww5NprtRWtoAg6Fo2LaBBgen2mlYKRe/r1Wg06FCp8OsnBw8NnYsPPE4OiIRHAoG8Z+9vTjS14cPQyG8lVgAAKjBRozFBWiQAKBAUYwwm+di/PhfMnhuANzzoWGREALeeBxno1G0BIPYF/DjfLAd9yj7UZ1zAVpNHkymr8FkuoszXBmMez404mgUBfk6HfJ1OtyWlYX/UVCA3kQxFFTCrOWJ5RsRw4eGXf9V2BaGzg2NT5YjIikYPkQkBcOHiKRg+BCRFAwfIpKC4UNEUjB8iEgKhg8RScHwISIpGD5EJAXDh4ikYPgQkRQMHyKSIq3wWbduHaZNmwaz2Qyz2QyHw4GdO3eq46FQCLW1tRgzZgxMJhMWLFgAt9ud9Ds6OjpQXV2NnJwcFBYWYtWqVYjFYoOzNUSUMdIKn/Hjx2PNmjVoaWnBwYMH8Y1vfAMPPvggjh07BgB48sknsW3bNmzevBnNzc3o6urC/Pnz1dfH43FUV1cjEolg79692LhxIzZs2IDVq1cP7lYR0cgnrlN+fr547bXXhMfjEXq9XmzevFkdO3HihAAgnE6nEEKIHTt2CI1GI1wul1qzbt06YTabRTgc/sL3CIVCwuv1qktnZ6cAILxe7/W2T0SDyOv1pvzZvOZzPvF4HA0NDQgEAnA4HGhpaUE0GkVlZaVaM3nyZJSUlMDpdAIAnE4npk6dCpvNptZUVVXB5/Ope08Dqa+vh8ViUZfi4uJrbZuIRoi0w6etrQ0mkwlGoxFPPPEEtmzZgrKyMrhcLhgMBlit1qR6m80Gl8sFAHC5XEnB0z/eP/ZF6urq4PV61aWzszPdtolohEn7Nqq33norWltb4fV68eabb6KmpgbNzc1D0ZvKaDTCaDQO6XsQ0fBKO3wMBgMmTpwIACgvL8eBAwfw0ksv4aGHHkIkEoHH40na+3G73bDb7QAAu92O/fv3J/2+/tmw/hoiujFc93U+iUQC4XAY5eXl0Ov1aGpqUsfa29vR0dEBh8MBAHA4HGhra0N3d7da09jYCLPZjLKysutthYgySFp7PnV1dbj//vtRUlICv9+PTZs24d1338U777wDi8WCxYsXY+XKlSgoKIDZbMayZcvgcDgwe/ZsAMDcuXNRVlaGRx55BGvXroXL5cIzzzyD2tpaHlYR3WDSCp/u7m784Ac/wNmzZ2GxWDBt2jS88847+OY3vwkAePHFF6HRaLBgwQKEw2FUVVXh1VdfVV+v1Wqxfft2LFmyBA6HA7m5uaipqcHzzz8/uFtFRCMen1hKRIMmnc8mv9tFRFIwfIhICoYPEUnB8CEiKRg+RCQFw4eIpGD4EJEUDB8ikoLhQ0RSMHyISAqGDxFJwfAhIikYPkQkBcOHiKRg+BCRFAwfIpKC4UNEUjB8iEgKhg8RScHwISIpGD5EJAXDh4ikYPgQkRQMHyKSguFDRFIwfIhICoYPEUnB8CEiKRg+RCQFw4eIpGD4EJEUDB8ikoLhQ0RSMHyISAqGDxFJcV3hs2bNGiiKghUrVqjrQqEQamtrMWbMGJhMJixYsAButzvpdR0dHaiurkZOTg4KCwuxatUqxGKx62mFiDLMNYfPgQMH8Lvf/Q7Tpk1LWv/kk09i27Zt2Lx5M5qbm9HV1YX58+er4/F4HNXV1YhEIti7dy82btyIDRs2YPXq1de+FUSUecQ18Pv9YtKkSaKxsVHcc889Yvny5UIIITwej9Dr9WLz5s1q7YkTJwQA4XQ6hRBC7NixQ2g0GuFyudSadevWCbPZLMLh8IDvFwqFhNfrVZfOzk4BQHi93mtpn4iGiNfrTfmzeU17PrW1taiurkZlZWXS+paWFkSj0aT1kydPRklJCZxOJwDA6XRi6tSpsNlsak1VVRV8Ph+OHTs24PvV19fDYrGoS3Fx8bW0TUQjSNrh09DQgEOHDqG+vv6yMZfLBYPBAKvVmrTeZrPB5XKpNZ8Pnv7x/rGB1NXVwev1qktnZ2e6bRPRCKNLp7izsxPLly9HY2MjsrKyhqqnyxiNRhiNxmF7PyIaemnt+bS0tKC7uxt33nkndDoddDodmpub8fLLL0On08FmsyESicDj8SS9zu12w263AwDsdvtls1/9P/fXENHol1b4zJkzB21tbWhtbVWXGTNmYNGiRep/6/V6NDU1qa9pb29HR0cHHA4HAMDhcKCtrQ3d3d1qTWNjI8xmM8rKygZps4hopEvrsCsvLw9TpkxJWpebm4sxY8ao6xcvXoyVK1eioKAAZrMZy5Ytg8PhwOzZswEAc+fORVlZGR555BGsXbsWLpcLzzzzDGpra3loRXQDSSt8UvHiiy9Co9FgwYIFCIfDqKqqwquvvqqOa7VabN++HUuWLIHD4UBubi5qamrw/PPPD3YrRDSCKUIIIbuJdPl8PlgsFni9XpjNZtntENFF6Xw2+d0uIpKC4UNEUjB8iEgKhg8RScHwISIpGD5EJAXDh4ikYPgQkRQMHyKSguFDRFIwfIhICoYPEUnB8CEiKRg+RCQFw4eIpGD4EJEUDB8ikoLhQ0RSMHyISAqGDxFJwfAhIikYPkQkBcOHiKRg+BCRFAwfIpKC4UNEUjB8iEgKhg8RScHwISIpGD5EJAXDh4ikYPgQkRQMHyKSguFDRFIwfIhICoYPEUnB8CEiKXSyG7gWQggAgM/nk9wJEX1e/2ey/zN6JRkZPhcuXAAAFBcXS+6EiAbi9/thsViuWJOR4VNQUAAA6OjouOoGjjQ+nw/FxcXo7OyE2WyW3U7K2PfwytS+hRDw+/0oKiq6am1Gho9G89mpKovFklH/Yz7PbDZnZO/se3hlYt+p7hDwhDMRScHwISIpMjJ8jEYjnnvuORiNRtmtpC1Te2ffwytT+06HIlKZEyMiGmQZuedDRJmP4UNEUjB8iEgKhg8RScHwISIpMjJ8XnnlFUyYMAFZWVmoqKjA/v37pfazZ88ePPDAAygqKoKiKNi6dWvSuBACq1evxk033YTs7GxUVlbi1KlTSTU9PT1YtGgRzGYzrFYrFi9ejN7e3iHtu76+HjNnzkReXh4KCwsxb948tLe3J9WEQiHU1tZizJgxMJlMWLBgAdxud1JNR0cHqqurkZOTg8LCQqxatQqxWGzI+l63bh2mTZumXv3rcDiwc+fOEd3zQNasWQNFUbBixYqM631QiAzT0NAgDAaD+P3vfy+OHTsmHnvsMWG1WoXb7ZbW044dO8TPfvYz8ac//UkAEFu2bEkaX7NmjbBYLGLr1q3iyJEj4jvf+Y4oLS0VfX19as19990npk+fLt5//33xt7/9TUycOFEsXLhwSPuuqqoSr7/+ujh69KhobW0V3/rWt0RJSYno7e1Va5544glRXFwsmpqaxMGDB8Xs2bPFV7/6VXU8FouJKVOmiMrKSnH48GGxY8cOMXbsWFFXVzdkff/5z38Wf/nLX8SHH34o2tvbxU9/+lOh1+vF0aNHR2zPl9q/f7+YMGGCmDZtmli+fLm6PhN6HywZFz6zZs0StbW16s/xeFwUFRWJ+vp6iV39l0vDJ5FICLvdLl544QV1ncfjEUajUbzxxhtCCCGOHz8uAIgDBw6oNTt37hSKoogzZ84MW+/d3d0CgGhublb71Ov1YvPmzWrNiRMnBADhdDqFEJ8Fr0ajES6XS61Zt26dMJvNIhwOD1vv+fn54rXXXsuInv1+v5g0aZJobGwU99xzjxo+mdD7YMqow65IJIKWlhZUVlaq6zQaDSorK+F0OiV29sVOnz4Nl8uV1LPFYkFFRYXas9PphNVqxYwZM9SayspKaDQa7Nu3b9h69Xq9AP7rrgEtLS2IRqNJvU+ePBklJSVJvU+dOhU2m02tqaqqgs/nw7Fjx4a853g8joaGBgQCATgcjozouba2FtXV1Uk9Apnx5z2YMupb7efPn0c8Hk/6gwcAm82GkydPSurqylwuFwAM2HP/mMvlQmFhYdK4TqdDQUGBWjPUEokEVqxYgbvuugtTpkxR+zIYDLBarVfsfaBt6x8bKm1tbXA4HAiFQjCZTNiyZQvKysrQ2to6YnsGgIaGBhw6dAgHDhy4bGwk/3kPhYwKHxo6tbW1OHr0KN577z3ZraTk1ltvRWtrK7xeL958803U1NSgublZdltX1NnZieXLl6OxsRFZWVmy25Euow67xo4dC61We9nZf7fbDbvdLqmrK+vv60o92+12dHd3J43HYjH09PQMy3YtXboU27dvx+7duzF+/Hh1vd1uRyQSgcfjuWLvA21b/9hQMRgMmDhxIsrLy1FfX4/p06fjpZdeGtE9t7S0oLu7G3feeSd0Oh10Oh2am5vx8ssvQ6fTwWazjdjeh0JGhY/BYEB5eTmamprUdYlEAk1NTXA4HBI7+2KlpaWw2+1JPft8Puzbt0/t2eFwwOPxoKWlRa3ZtWsXEokEKioqhqw3IQSWLl2KLVu2YNeuXSgtLU0aLy8vh16vT+q9vb0dHR0dSb23tbUlhWdjYyPMZjPKysqGrPdLJRIJhMPhEd3znDlz0NbWhtbWVnWZMWMGFi1apP73SO19SMg+452uhoYGYTQaxYYNG8Tx48fF448/LqxWa9LZ/+Hm9/vF4cOHxeHDhwUA8atf/UocPnxYfPzxx0KIz6barVareOutt8QHH3wgHnzwwQGn2u+44w6xb98+8d5774lJkyYN+VT7kiVLhMViEe+++644e/asugSDQbXmiSeeECUlJWLXrl3i4MGDwuFwCIfDoY73T/3OnTtXtLa2irfffluMGzduSKd+n376adHc3CxOnz4tPvjgA/H0008LRVHEX//61xHb8xf5/GxXpvV+vTIufIQQ4je/+Y0oKSkRBoNBzJo1S7z//vtS+9m9e7cAcNlSU1MjhPhsuv3ZZ58VNptNGI1GMWfOHNHe3p70Oy5cuCAWLlwoTCaTMJvN4tFHHxV+v39I+x6oZwDi9ddfV2v6+vrEj3/8Y5Gfny9ycnLEd7/7XXH27Nmk3/PPf/5T3H///SI7O1uMHTtWPPXUUyIajQ5Z3z/60Y/EzTffLAwGgxg3bpyYM2eOGjwjtecvcmn4ZFLv14v38yEiKTLqnA8RjR4MHyKSguFDRFIwfIhICoYPEUnB8CEiKRg+RCQFw4eIpGD4EJEUDB8ikoLhQ0RS/H+Or78vertMRAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-63.0 64 [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, 0.0]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        action = action.item()\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 2 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
